/**
 * @date 2015年12月28日 下午9:33:51
 * @version V1.0
 */
package edu.cn.nlsde.tmfst.processor;

import edu.cn.nlsde.Utils.IOUtil;
import edu.cn.nlsde.Utils.Utils;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;

/**
 * @author ffftzh
 * @ClassName: data_pre_process
 * @Description: 用于预处理文档，去除非英文单词和低高频词,以及去除句子长度过短或者过长的句子
 * @date 2015年12月28日 下午9:33:51
 */
public class data_pre_process_emnlp {
    private HashMap<String, Integer> wordCount = new HashMap<String, Integer>();
    private HashSet<String> stopList = new HashSet<String>();
    private ArrayList<String> tag_list = null;
    private int frequency_num = 0;
    private int min_sentence_length = 0;
    private int max_sentence_length = 0;
    private String path = "";
    private String tag_path = "";
    private ArrayList<ArrayList<String>> corpus = new ArrayList<>();
    private HashMap<String, String> emnlpMap;

    public data_pre_process_emnlp(String path, String tag_path, int frequency_num, int min_sentence_length, int max_sentence_length) throws IOException {
        this.frequency_num = frequency_num;
        this.path = path;
        this.tag_path = tag_path;
        this.min_sentence_length = min_sentence_length;
        this.max_sentence_length = max_sentence_length;

        BufferedReader reader = IOUtil.getReader("common/stopwords_big");
        String line = null;
        while ((line = reader.readLine()) != null) {
            line = line.trim();
            stopList.add(line.toLowerCase());
        }
        reader.close();
    }

    private void read_tag() throws IOException {
        this.tag_list = new ArrayList<String>();
        BufferedReader reader = IOUtil.getReader(this.tag_path);
        String line = null;
        while ((line = reader.readLine()) != null) {
            tag_list.add(line);
        }
        reader.close();

    }

    private void read_emnlp() throws IOException {
        this.emnlpMap = new HashMap<String, String>();
        BufferedReader reader = IOUtil.getReader("common/emnlp_dict.txt");
        String line = null;
        while ((line = reader.readLine()) != null) {
            String[] lines = line.split("\\s+");
            this.emnlpMap.put(lines[0], lines[1]);
        }
        reader.close();
    }


    public static List<String> toWordList(String sentence) {
        ArrayList<String> result = new ArrayList<>();
        String[] list = sentence.split("[^0-9a-zA-Z]+");
        for (String item : list) {
            result.add(item);
        }
        return result;
    }

    private void read_data() throws IOException {
        if (this.tag_path != null) {
            read_tag();
        }

        BufferedReader reader = IOUtil.getReader(this.path);
        String line = null;
        while ((line = reader.readLine()) != null) {
            line = line.toLowerCase();
            ArrayList<String> newLine = new ArrayList<>();

            if (line.trim().length() > 4) {
//                Sentence sent = new Sentence(line);
//                List<String> postagList = sent.posTags();
//                List<String> lemmasList = sent.lemmas();
                List<String> words = toWordList(line);
//                assert postagList.size() == lemmasList.size();
                for (int index = 0; index < words.size(); index++) {
//                    if (!(postagList.get(index).startsWith("N")
//                            || postagList.get(index).startsWith("J")
//                            || postagList.get(index).startsWith("R")
//                            || postagList.get(index).startsWith("V")
//                    )) {
//                        continue;
//                    }
                    String tWord = words.get(index);
                    if (this.emnlpMap.containsKey(tWord)) {
                        tWord = this.emnlpMap.get(tWord);
                    }
//                    String tWord = lemmasList.get(index);

                    if (tWord.matches(".*[^0-9a-zA-Z]+.*")
                            || tWord.length() < 3
                            || this.stopList.contains(tWord)) {
                        continue;
                    }

                    if (!wordCount.containsKey(tWord)) {
                        wordCount.put(tWord, 0);
                    }
                    wordCount.put(tWord, wordCount.get(tWord) + 1);
                    newLine.add(tWord);
                }
            }
            this.corpus.add(newLine);
        }
        reader.close();

    }

    private void write_data() throws IOException {

        BufferedWriter writer = IOUtil.getWriter(path + ".processed3");
        BufferedWriter tag_writer = IOUtil.getWriter(tag_path + ".processed3");
        ArrayList<String> write_line = new ArrayList<String>();
        if (tag_path != null) {
            assert corpus.size() == tag_list.size();
        }
        for (ArrayList<String> oneDoc : this.corpus) {
            write_line.clear();

            for (String item : oneDoc) {
                if (wordCount.get(item) < this.frequency_num) {
                    continue;
                }
                write_line.add(item);
            }
            String oneTagString = "";
            if (tag_path != null) {
                oneTagString = tag_list.remove(0);
            }
            if (write_line.size() > min_sentence_length && write_line.size() < max_sentence_length) {
                writer.write(Utils.join(write_line, " ") + "\n");
                tag_writer.write(oneTagString + "\n");
            }
        }
        if (this.tag_path != null && tag_list.size() > 0) {
            throw new IOException("tag num error！");
        }

        tag_writer.close();
        writer.close();
    }

    public static void pre_process_data(String path, String tagPath, int frequency, int min_sentence_length, int max_sentence_length) throws IOException {
        data_pre_process_emnlp instance = new data_pre_process_emnlp(path, tagPath, frequency, min_sentence_length, max_sentence_length);
        instance.read_emnlp();
        instance.read_data();
        instance.write_data();
    }


    public static void main(String[] args) throws IOException {
        String root = "G:\\pycharm\\TopicModelForShortText_py\\data\\tweet_without_normalize\\";
        pre_process_data(root + "new-tweet.data", root + "new-tweet.tag", 10, 2, 20);

    }

}
